from scrapyProject.spiders.spiderBase import spiderBase
from scrapyProject.spiders.fang.fangDataBase import fangDataBase
from scrapyProject.spiders.fang.Items import CatalogItem, HouseInfoItem
import time


class fang_spider(spiderBase):
    name = 'fang'
   # type = ["Dynamic"]
    def start_requests(self):
        dataBase = fangDataBase()

        isHasData= False
        timeColl=time.time()
        for id,urlNew,urlSecond,urlRental in dataBase.getNeedDownUrls():
            isHasData = True
            yield self.RequestUrl(url=urlNew, callback=self.parseNewHouse,      meta={'houseType': 0, 'idCatalog':id, 'collTime':timeColl}, isMustParse=True)
            yield self.RequestUrl(url=urlSecond, callback=self.parseSecondHouse,   meta={'houseType': 1, 'idCatalog':id, 'collTime':timeColl}, isMustParse=True)
            yield self.RequestUrl(url=urlRental, callback=self.parseRentalHouse,   meta={'houseType': 2, 'idCatalog':id, 'collTime':timeColl}, isMustParse=True)

        if not isHasData:
            urls=["https://www.fang.com/SoufunFamily.htm"]
            for url in urls:
                yield self.RequestUrl(url=url, callback=self.parseCatalog, isMustParse=True)

    def parseCatalog(self, response):
        verData = response.xpath('//div[@class="outCont"]//tr')
        strProviceName=""
        for trData in verData:
            strProviceNameTemp=trData.xpath('.//strong/text()').extract()
            if len(strProviceNameTemp)!=0 :
                s = strProviceNameTemp[0].strip()
                if len(s) > 0 :
                    strProviceName = s
            xpathCityName=trData.xpath('.//a')
            vStrCityname=xpathCityName.xpath('./text()').extract()
            vUrlCity=xpathCityName.xpath('./@href').extract()
            for i in range(len(vStrCityname)):
                itemData=CatalogItem()
                itemData['CityName'] = vStrCityname[i]
                itemData['ProvinceName'] = strProviceName
                itemData['CollectTime'] = time.time()

                yield self.RequestUrl(url=self.constructUrl(response, vUrlCity[i]), callback=self.parseCatalogItem, isMustParse=True, meta={'item': itemData})

    def parseCatalogItem(self, response):
        itemData=response.meta['item']
        itemData['urlNewHouse'] = self.constructUrl(response, response.xpath( '//a[@id="dsy_D02_02"]/@href').extract()[0])
        itemData['urlSencondHandHouse'] =self.constructUrl(response,  response.xpath( '//a[@id="dsy_D02_03"]/@href').extract()[0])
        itemData['urlRentalhousing'] = self.constructUrl(response, response.xpath('//a[@id="dsy_D02_04"]/@href').extract()[0])
        yield itemData

    def TrimStr(self, str):
        str=str.replace("\t", "")
        str = str.replace("\r", "")
        str = str.replace("\n", "")
        str=str.strip()
        return  str

    def parseNewHouse(self, response):
        veXPathBase = response.xpath( '//div[@class="nl_con clearfix"]//div[@class="nlc_details"]')
        for xPathBase in veXPathBase:
            houseName = xPathBase.xpath('string(.//div[contains(@class,"name")]/a)').extract()
            houseType=xPathBase.xpath('string(.//div[contains(@class,"type")])').extract()
            houseAddress = xPathBase.xpath('string(.//div[contains(@class,"address")])').extract()
            houseTele= xPathBase.xpath('string(.//div[contains(@class,"tel")])').extract()
            housecharacteristic = xPathBase.xpath('string(.//div[contains(@class,"fangyuan")])').extract()
            housePrice = xPathBase.xpath('string(.//div[contains(@class,"price")])').extract()
            itemInfo = HouseInfoItem()
            itemInfo["name"] = self.TrimStr(houseName[0]) if len(houseName) else ""
            itemInfo["housetype"] = self.TrimStr(houseType[0]) if len(houseType) else ""
            itemInfo["address"] = self.TrimStr(houseAddress[0]) if len(houseAddress) else ""
            itemInfo["tele"] = self.TrimStr(houseTele[0]) if len(houseTele) else ""
            itemInfo["characteristic"] = self.TrimStr(housecharacteristic[0]) if len(housecharacteristic) else ""
            itemInfo["price"] = self.TrimStr(housePrice[0]) if len(housePrice) else ""
            itemInfo["type"] = response.meta["houseType"]
            itemInfo["idCatalog"] = response.meta["idCatalog"]
            itemInfo["collecttime"] = response.meta["collTime"]
            yield itemInfo

        lsNextPage=response.xpath('//a[@class="active"]/following-sibling::a[1]/@href').extract()
        if len(lsNextPage) > 0:
            url = self.constructUrl(response, lsNextPage[0])
            yield self.RequestUrl(url=url, callback=self.parseNewHouse,
                                  isMustParse=True, meta={'houseType': response.meta["houseType"], 'idCatalog':response.meta["idCatalog"], 'collTime':response.meta["collTime"]})

    def parseSecondHouse(self, response):
        veXPathBase = response.xpath('//dl[@dataflag and @data-bg]')
        for xPathBase in veXPathBase:
            houseName = xPathBase.xpath('string(.//h4)').extract()
            houseType = xPathBase.xpath('string(.//p[contains(@class,"tel_shop")])').extract()
            houseAddress = xPathBase.xpath('string(.//p[contains(@class,"add_shop")])').extract()
            houseTele = [""]
            housecharacteristic = xPathBase.xpath('string(.//p[contains(@class,"clearfix label")])').extract()
            housePrice = xPathBase.xpath('string(.//dd[contains(@class,"price_right")])').extract()
            itemInfo = HouseInfoItem()
            itemInfo["name"] = self.TrimStr(houseName[0]) if len(houseName) else ""
            itemInfo["housetype"] = self.TrimStr(houseType[0]) if len(houseType) else ""
            itemInfo["address"] = self.TrimStr(houseAddress[0]) if len(houseAddress) else ""
            itemInfo["tele"] = self.TrimStr(houseTele[0]) if len(houseTele) else ""
            itemInfo["characteristic"] = self.TrimStr(housecharacteristic[0]) if len(housecharacteristic) else ""
            itemInfo["price"] = self.TrimStr(housePrice[0]) if len(housePrice) else ""
            itemInfo["type"] = response.meta["houseType"]
            itemInfo["idCatalog"] = response.meta["idCatalog"]
            itemInfo["collecttime"] = response.meta["collTime"]
            yield itemInfo

        lsNextPage = response.xpath('//span[@class = "on"]/following-sibling::span[1]/a/@href').extract()
        if len(lsNextPage) > 0:
            url = self.constructUrl(response, lsNextPage[0])
            yield self.RequestUrl(url=url, callback=self.parseSecondHouse,
                                  isMustParse=True, meta={'houseType': response.meta["houseType"],
                                                          'idCatalog': response.meta["idCatalog"],
                                                          'collTime': response.meta["collTime"]})

    def parseRentalHouse(self, response):
        veXPathBase = response.xpath('//dd[@class="info rel"]')
        for xPathBase in veXPathBase:
            houseName = xPathBase.xpath('string(.//p[@class="title"])').extract()
            houseType = xPathBase.xpath('string(.//p[@class="title"]/following-sibling::p[1])').extract()
            houseAddress = xPathBase.xpath('string(.//p[@class="title"]/following-sibling::p[2])').extract()
            houseTele = [""]
            housecharacteristic = xPathBase.xpath('string(.//p[@class="mt12"])').extract()
            housePrice = xPathBase.xpath('string(.//div[@class="moreInfo"]//span[contains(@class,"price")]/..)').extract()
            itemInfo = HouseInfoItem()
            itemInfo["name"] = self.TrimStr(houseName[0]) if len(houseName) else ""
            itemInfo["housetype"] = self.TrimStr(houseType[0]) if len(houseType) else ""
            itemInfo["address"] = self.TrimStr(houseAddress[0]) if len(houseAddress) else ""
            itemInfo["tele"] = self.TrimStr(houseTele[0]) if len(houseTele) else ""
            itemInfo["characteristic"] = self.TrimStr(housecharacteristic[0]) if len(housecharacteristic) else ""
            itemInfo["price"] = self.TrimStr(housePrice[0]) if len(housePrice) else ""
            itemInfo["type"] = response.meta["houseType"]
            itemInfo["idCatalog"] = response.meta["idCatalog"]
            itemInfo["collecttime"] = response.meta["collTime"]
            yield itemInfo

        lsNextPage = response.xpath('//a[@class = "pageNow"]/following-sibling::a[1]/@href').extract()
        if len(lsNextPage) > 0:
            url = self.constructUrl(response, lsNextPage[0])
            yield self.RequestUrl(url=url, callback=self.parseRentalHouse,
                                  isMustParse=True, meta={'houseType': response.meta["houseType"],
                                                          'idCatalog': response.meta["idCatalog"],
                                                          'collTime': response.meta["collTime"]})